#!/usr/bin/python3
# -*- coding:utf-8 -*-
from bs4 import BeautifulSoup as bs
from urllib import request
from time import strftime, time, sleep
from db import Db
import re
import gevent
from gevent import monkey
monkey.patch_all()
__author__ = 'bool'

class App(object):

    # init 初始化函数
    def __init__(self):
        pass

    # 返回html
    def getHtml(self, url):
        try:
            resp = request.urlopen(url).read().decode("gbk")
            return bs(resp, "html.parser")
        except IOError:
            print('网络异常: '+url)
        else:
            pass

    # 执行
    def run(self, url):
        soup = self.getHtml(url)
        self.getList(soup)

    # 获取 省份 直辖市 列表
    def getList(self, soup=None, pid=0, code=0):
        # 选中有内容的区域
        res = soup.findAll("tr", {"class": "provincetr"})

        # 线程列表
        thread = []
        # 遍历 行数
        for item in res:

            # 遍历行数
            for line in item:
                # 这里获取的是 td 里面包裹的内容
                a_label = line.find("a")
                # 判断是否出现 NoneType
                if a_label is None:
                    pass
                else:
                    temp = {
                        'title': a_label.getText(),
                        'pid': pid,
                        'code': code,
                        'href': 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + a_label['href'],
                        'upTime': time(),
                        'upDate': strftime("%Y-%m-%d %H:%M:%S"),
                        'level': 1
                    }

                    urlcode = a_label['href'].replace('.html', '')
                    id = self.insertData(temp)
                    # 获取城市
                    # self.getCity(temp['href'], id, urlcode)
                    # 加入到 线程池
                    thread.append(gevent.spawn(self.getCity, temp['href'], id, urlcode))
        # 开启协程
        gevent.joinall(thread)


    # 获取 城市 列表
    def getCity(self, url=None, pid=0, urlcode='11'):

        soup = self.getHtml(url)
        # 选中有内容的区域
        res = soup.findAll("tr", {"class": "citytr"})

        # 遍历 行数
        for item in res:
            #这里获取的是 td 里面包裹的内容
            line = item.findAll("td")
            # 名称 a标签
            a_label = line[1].find('a')
            # 判断是否出现 NoneType
            if a_label is None:
                pass
            else:
                temp = {
                    'title': a_label.getText(),
                    'pid': pid,
                    'code': line[0].find('a').getText(),
                    'href': 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + a_label['href'],
                    'upTime': time(),
                    'upDate': strftime("%Y-%m-%d %H:%M:%S"),
                    'level': 2
                }

                id = self.insertData(temp)
                # 获取 县区
                self.getCounty(temp['href'], id, urlcode)

    # 获取 县区 列表
    def getCounty(self, url=None, pid=0, urlcode='11'):
        soup = self.getHtml(url)
        # 选中有内容的区域
        res = soup.findAll("tr", {"class": "countytr"})

        # 遍历 行数
        for item in res:
            #这里获取的是 td 里面包裹的内容
            line = item.findAll("td")
            # 名称
            a_label = line[1].find('a')
            # 判断是否出现 NoneType
            if a_label is None:
                pass
            else:
                temp = {
                    'title': a_label.getText(),
                    'pid': pid,
                    'code': line[0].find('a').getText(),
                    'href': 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + urlcode + '/' + a_label['href'],
                    'upTime': time(),
                    'upDate': strftime("%Y-%m-%d %H:%M:%S"),
                    'level': 3
                }
                countyCode = re.findall(r'^\d{1,3}', a_label['href'])[0]
                id = self.insertData(temp)
                # 获取 乡镇
                self.getTown(temp['href'], id, urlcode, countyCode)


    # 获取 乡镇 列表
    def getTown(self, url=None, pid=0, urlcode='11', countyCode='01'):
        soup = self.getHtml(url)
        # 选中有内容的区域
        res = soup.findAll("tr", {"class": "towntr"})

        # 遍历 行数
        for item in res:
            #这里获取的是 td 里面包裹的内容
            line = item.findAll("td")
            # 名称
            a_label = line[1].find('a')
            # 判断是否出现 NoneType
            if a_label is None:
                pass
            else:
                temp = {
                    'title': a_label.getText(),
                    'pid': pid,
                    'code': line[0].find('a').getText(),
                    'href': 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + urlcode + '/' + countyCode + '/' +a_label['href'],
                    'upTime': time(),
                    'upDate': strftime("%Y-%m-%d %H:%M:%S"),
                    'level': 4
                }
                id = self.insertData(temp)
                # 获取 村社区
                self.getVillagetr(temp['href'], id)

    # 获取 村社区 列表
    def getVillagetr(self, url=None, pid=0, urlcode='11'):
        soup = self.getHtml(url)
        # 选中有内容的区域
        res = soup.findAll("tr", {"class": "villagetr"})

        # 遍历 行数
        for item in res:
            #这里获取的是 td 里面包裹的内容
            line = item.findAll("td")
            temp = {
                'title': line[2].getText(),
                'pid': pid,
                'code': line[0].getText(),
                'href': url,
                'upTime': time(),
                'upDate': strftime("%Y-%m-%d %H:%M:%S"),
                'level': 5
            }
            self.insertData(temp)

    # 插入数据
    def insertData(self, temp):
        sqls = "SELECT * FROM main WHERE href = '"+temp['href']+"' AND pid = "+str(temp['pid'])+" AND code = '"+str(temp['code'])+"' "
        db = Db()
        findRes = db.dml(sqls)
        if findRes:
            print("id:"+str(findRes[0][0])+' - '+temp['title']+" --------> 已经存在")
            return findRes[0][0]
        else:
            sqls = "insert into main ('title','pid','code','href','upTime','upDate', 'level') values ('%s','%s','%s','%s','%s','%s','%s')" % (
            temp['title'],  temp['pid'], temp['code'], temp['href'], temp['upTime'], temp['upDate'], temp['level'])
            db = Db()
            id = db.ddl(sqls)
            if id > 0:
                print("id:"+str(id)+' - '+temp['title']+' --------> 采集成功')
                return id

if __name__ == '__main__':
    api = App()
    api.run('http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html')



